package com.zxl.test;

import java.io.File;
import java.io.FileOutputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;

public class Get51job {
	
	public static String subNode = "";
	public static String zwStr = "软件工程师,软件开发工程师,Java软件工程师,java软件工程师,Java工程师,java工程师,软件开发";

	public static void main(String[] args) {
		Long st = System.currentTimeMillis();
		String filePath = "D:\\job.txt";
		
		int pageNum = 20;
		Random r = new Random();
		for(int j = 1; j <= pageNum; j++){
			String url = "http://search.51job.com/list/010000%252C00,%2B,%2B,%2B,%2B,%2B,java%25B9%25A4%25B3%25CC%25CA%25A6,2," + j + ".html";
			Set<String> rt = getNodeList(url);
			Iterator it = rt.iterator();
			System.out.println(rt.size());
			int n = 0;
			while(n <= 5){
				n++;
				Object obj = it.next();
				System.out.println(obj.toString());
				String company = enCode(obj.toString());
				String url_fz = "http://www.fenzhi.com/index.php?name=" + company + "&class=xinshui&dz=search";
//				System.out.println(url_fz);
				Map map = getNodeList_fz(url_fz);
				ArrayList<String> occ = (ArrayList<String>) map.get("occ");
				ArrayList<String> mean = (ArrayList<String>) map.get("mean");
				ArrayList<String> lowValue = (ArrayList<String>) map.get("lowValue");
				ArrayList<String> highValue = (ArrayList<String>) map.get("highValue");
//				System.out.println(occ.size() + " " + mean.size() + " " + lowValue.size() + " " + highValue.size());
				System.out.println(mean);
				if(mean.size() > 1){
					StringBuffer sb = new StringBuffer();
					sb.append("===============\n");
					sb.append(obj.toString() + "\n"); //公司名称
				
					for(int z = 0; z < lowValue.size(); z++){
						if(mean.size() > lowValue.size() && lowValue.size() == highValue.size()){
							if(zwStr.indexOf((occ.get(z))) != -1){
								sb.append(occ.get(z) + "\t" + mean.get(z+1) + "\t" + lowValue.get(z) + "\t" + highValue.get(z) + "\n");
							}
						}else{
							if(zwStr.indexOf((occ.get(z))) != -1){
								sb.append(occ.get(z) + "\t" + mean.get(z+1) + "\n");
							}
						}
					}
					writeFile(filePath, sb);
				}
				try {
					int s = r.nextInt(5) + 5;
					Thread.sleep(s * 1000);
				} catch (InterruptedException e) {
					e.printStackTrace();
				}
			}
		}
		Long et = System.currentTimeMillis();
		System.out.println("共耗时" + (et-st)/1000 + "秒");
	}

	/**
	 * 51job
	 * @param url
	 * @return
	 */
	public static Set<String> getNodeList(String url) {
//		final ArrayList<String> result = new ArrayList<String>();
		final Set<String> result = new HashSet();
		Parser parser = null;
		NodeList nodeList = null;
		
		try {
			parser = new Parser(url);
			parser.setEncoding("GBK");
			nodeList = parser.parse(new NodeFilter() {
				@Override
				public boolean accept(Node node) {
					Node need = node;
					if (getStringsByRegex(node.getText(),"td class=\"td2\"")) {
						if(subNode.equals(need.toPlainTextString())){
							
						}else{
							subNode = need.toPlainTextString();
						result.add(need.toPlainTextString());
						need = need.getPreviousSibling()
								.getPreviousSibling();
						return true;
						}
					}
					return false;
				}
			});
		} catch (ParserException e) {
			e.printStackTrace();
		}
		return result;
	}

	/**
	 * 从分智网获取某公司的职位、平均、最低、最高薪水
	 * @param url
	 * @return
	 */
	public static Map getNodeList_fz(String url) {
		Map map = new HashMap();
		final ArrayList<String> occ = new ArrayList<String>();
		final ArrayList<String> mean = new ArrayList<String>();
		final ArrayList<String> lowValue = new ArrayList<String>();
		final ArrayList<String> highValue = new ArrayList<String>();
		Parser parser = null;
		NodeList nodeList = null;
		
		try {
			parser = new Parser(url);
			parser.setEncoding("UTF-8");
			nodeList = parser.parse(new NodeFilter() {
				@Override
				public boolean accept(Node node) {
					Node need = node;
					if (getStringsByRegex(node.getText(),"p class='link'")) {
						occ.add(need.toPlainTextString());
					}
					if (getStringsByRegex(node.getText(),"td class='mean'")) {
						mean.add(need.toPlainTextString());
					}
					if (getStringsByRegex(node.getText(),"div class='lowValue'")) {
						lowValue.add(need.toPlainTextString());
					}
					if (getStringsByRegex(node.getText(),"div class='highValue'")) {
						highValue.add(need.toPlainTextString());
					}
					return true;
				}
			});
		} catch (ParserException e) {
			e.printStackTrace();
		}
		map.put("occ", occ);
		map.put("mean", mean);
		map.put("lowValue", lowValue);
		map.put("highValue", highValue);
		return map;
	}
	
	/**
	 * 从网页中获取是否有匹配项
	 * @param txt
	 * @param regex
	 * @return
	 */
	public static boolean getStringsByRegex(String txt, String regex) {
		Pattern p = Pattern.compile(regex);
		Matcher m = p.matcher(txt);
		if (m.find()) {
			return true;
		}
		return false;
	}
	
	public static String enCode(String a){
		String b = "";
	    try {  
	        b = URLEncoder.encode(a,"UTF-8");  
	    } catch (UnsupportedEncodingException e) {  
	        e.printStackTrace();  
	    }
	    return b;
	}
	
	/**
	 * 把内容写到文件
	 * @param filePath
	 * @param strBuff
	 */
	public static void writeFile(String filePath, StringBuffer strBuff){
		try {
			File file = new File(filePath);
	        OutputStream out = new FileOutputStream(file,true);  
	        byte[] b = strBuff.toString().getBytes();  
	        for (int i = 0; i < b.length; i++) {  
	            out.write(b[i]);  
	        }
	        out.close();
	    } catch (Exception e) {
	    	e.printStackTrace();
	    }
	}
}
